package controller;

import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.logging.Logger;

import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;


import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.NativeArray;
import org.mozilla.javascript.Scriptable;
import org.mozilla.javascript.ScriptableObject;

import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPMethod;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.appengine.api.urlfetch.URLFetchServiceFactory;
import com.google.gson.Gson;
import com.xty.Document;
import com.xty.Elements;
import com.xty.Xsoup;


@SuppressWarnings("serial")
public class HatforentAppengineUrlWorkerServlet extends HttpServlet {
	//Logger log  = Logger.getLogger(HatforentAppengineUrlWorkerServlet.class.getName());
	
	public void doPost(HttpServletRequest req, HttpServletResponse resp)
			throws IOException {
		resp.setContentType("text/plain");
		
		String sitemap = req.getParameter("sitemap");
		String blog = req.getParameter("blog");
		String reglink = req.getParameter("linkSelecter");
		String type = req.getParameter("type");
		String tag = req.getParameter("tag");
		String linkSelecterJs = req.getParameter("linkSelecterJs");
		String activeLinkJs = req.getParameter("activeLinkJs");
		String getAll = req.getParameter("getAll");
		String linkSelecter = req.getParameter("linkSelecter");
		
		String titleSelecter = req.getParameter("titleSelecter");
		String contentSelecter = req.getParameter("contentSelecter");
		String removeSelecter = req.getParameter("removeSelecter");
		String customSelecter = req.getParameter("customSelecter");
		String category = req.getParameter("category");
		String typelink = req.getParameter("typelink");
		String source = req.getParameter("source");
		String sourcetype = req.getParameter("sourcetype");
		String source_image = req.getParameter("source_image");
		String remove_tag_a = req.getParameter("remove_tag_a");
		String activeContentJs = req.getParameter("activeContentJs");
		String userBlog = req.getParameter("userBlog");
		String arrLinkString = req.getParameter("arrLink");
		
		ArrayList<String> arrLink2 = new ArrayList<String>();
		
		String[] ar = arrLinkString.split("\\|");
		
		for (int i = 0; i < ar.length; i++) {
			arrLink2.add(ar[i]);
		}
		
		sitemap = Utils.fomatSitemap(sitemap);
		
		String charset="UTF-8";
		//
		Document doc = null;
		
		ArrayList<String> site = Utils.getStachSite();
		
		ArrayList<String> arrLink = new ArrayList<String>();
		
		if(activeLinkJs!=null && !activeLinkJs.trim().equals("yes"))
		{
			try {
				if(!site.contains(Utils.fomatDmain(sitemap)))
				{
					if(type.equals("html"))
					{
						doc = Xsoup.parse(Utils.GetHtml(sitemap, charset));
						
						if(getAll!=null&&getAll.equals("yes"))
						{
							for (int i = 0; i < doc.select(reglink).size(); i++) {
								arrLink.add(Utils.fomatLink(doc.select(reglink).get(i).attr("href"),sitemap));
							}
						} else {
							arrLink.add(Utils.fomatLink(doc.select(reglink).get(0).attr("href"),sitemap));
						}
					}
					else
					{
						if(getAll!=null&&getAll.equals("yes"))
						{
							try {
								doc = Xsoup.parse(Utils.GetHtml(sitemap, charset).replaceAll("<link>", "<links>").replaceAll("</link>", "</links>"));

//								arrLink.add(Utils.fomatLink(doc.select("item").get(0).select("links").get(0).text(),sitemap));
								for (int i = 0; i < doc.select("item").size(); i++) {
									arrLink.add(Utils.fomatLink(doc.select("item").get(i).select("links").get(0).text(),sitemap));
								}
							} catch (Exception e) {
								arrLink.add(Utils.fomatLink(doc.select("entry link[rel=alternate]").attr("href"),sitemap));
							}
						} else {
							try {
								doc = Xsoup.parse(Utils.GetHtml(sitemap, charset).replaceAll("<link>", "<links>").replaceAll("</link>", "</links>"));

								arrLink.add(Utils.fomatLink(doc.select("item").get(0).select("links").get(0).text(),sitemap));
								
							} catch (Exception e) {
								arrLink.add(Utils.fomatLink(doc.select("entry link[rel=alternate]").attr("href"),sitemap));
							}
						}
						
						
					}
					
//					link=Utils.fomatLink(link, sitemap);
				} else {
					String str = Utils.GetHtml("https://api.stackexchange.com/2.1/search/advanced?order=desc&answers=1&sort=activity&tagged="+tag+"&site="+Utils.fomatDmain(sitemap)+"&filter=!T5TVZso8yZqn*VSo%29T&key=vxwi93Z)iv36250wzn6aVA((", "utf-8");
					
					Gson gson = new Gson();
					
					Questions question = gson.fromJson(str, Questions.class);
					
					arrLink.add(Utils.fomatLink(question.items.get(0).getLink(),sitemap));
					
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
		} else {
			try {
				doc = Xsoup.parse(Utils.GetHtml(sitemap, charset)); 
				
				Context context = Context.enter();
				
				linkSelecterJs = linkSelecterJs+" var link = doGet(document);";
				
				ScriptableObject scope = context.initStandardObjects();
			    Scriptable that = context.newObject(scope);
			    Function fct = context.compileFunction(scope, linkSelecterJs, "script", 1, null);
			    Object result = fct.call(
			            context, scope, that, new Object[] {doc,sitemap});
			    
			    NativeArray arr = (NativeArray) Context.jsToJava(result, NativeArray.class);
			    for (Object o : arr.getIds()) {
			        String index1 = (String)o;
			        
			        if(index1.equals("link"))
			        {
				        Elements elements = (Elements) arr.get(index1);
				        				       
				        if(getAll!=null&&getAll.equals("yes"))
						{
							for (int j = 0; j < elements.size(); j++) {
								arrLink.add(Utils.fomatLink(elements.get(j).attr("href"),sitemap));
							} 
						} else {
							arrLink.add(Utils.fomatLink(elements.get(0).attr("href"),sitemap));
						}
			        }
			        if(index1.equals("page"))
			        {
			        	URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
						
						URL url = new URL("http://hatforrent-home.appspot.com/update-url");
						
						try {
							HTTPRequest request = new HTTPRequest(url, HTTPMethod.POST);
							
				            request.setHeader(new HTTPHeader("Content-Type", "application/x-www-form-urlencoded"));
				            String params = "link="+ Utils.EncodeString(arr.get(index1).toString()) +
				            				"&blog="+ Utils.EncodeString(blog)+
				            				"&sitemap="+ Utils.EncodeString(sitemap);
				            request.setPayload(params.getBytes());
				            
				            fetcher.fetchAsync(request);
				            
						} catch (Exception e) {
							e.printStackTrace();
						}
			        }
			    }
				
			} catch (Exception e) {
				e.printStackTrace();
			}
		}

		if(arrLink.size()<=0)
		{
			String error = "Selector error";
			URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
			try {
				URL url = new URL("http://hatforrent-home.appspot.com/update-source-error");
				try {
					HTTPRequest request1 = new HTTPRequest(url, HTTPMethod.POST);
					
		            request1.setHeader(new HTTPHeader("Content-Type", "application/x-www-form-urlencoded"));
		            String params = "sitemap="+ sitemap +
		            				"&userBlog="+ userBlog +
		            				"&blog="+ blog +
		            				"&error="+ error;
		            request1.setPayload(params.getBytes());
		            
		            fetcher.fetchAsync(request1);
				} catch (Exception e) {
					e.printStackTrace();
				}
				
			} catch (Exception e) {
				e.printStackTrace();
			}
			
		}
		
		Object[] st = arrLink.toArray();
		for (Object s : st) {
			if (arrLink.indexOf(s) != arrLink.lastIndexOf(s)) {
				arrLink.remove(arrLink.lastIndexOf(s));
			}
		}
//		log.warning(arrLink.size()+"|"+sitemap+"|"+arrLink2.size()+"|"+arrLink2.get(0)+"|"+arrLink.get(0));
		arrLink.removeAll(arrLink2);
		
		if(getAll!=null&&getAll.equals("yes")&&arrLink.size()<=5)
		{
			URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
			
			URL url = new URL("http://hatforrent-home.appspot.com/change-time-craw");
			
			try {
				HTTPRequest request = new HTTPRequest(url, HTTPMethod.POST);
				
	            request.setHeader(new HTTPHeader("Content-Type", "application/x-www-form-urlencoded"));
	            String params = "userBlog="+ Utils.EncodeString(userBlog) +
	            				"&blog="+ Utils.EncodeString(blog)+
	            				"&sitemap="+ Utils.EncodeString(sitemap);
	            request.setPayload(params.getBytes());
	            
	            fetcher.fetchAsync(request);
	            
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		
		
		
		if(arrLink.size()>0)
		{
			URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
			
			URL url = new URL("http://hatforrent-home.appspot.com/get-link-worker-url");
			
			String listLink = "";
			
			for (int i = 0; i < arrLink.size(); i++) {
				String link = Utils.fomatLink(arrLink.get(i), sitemap);
				if(link!=null&&!link.trim().equals(""))
				{
					if(listLink.equals(""))
						listLink = link;
					else
						listLink += "|"+link;
				}
				if(arrLink2.size()>=50)
					arrLink2.remove(0);
				arrLink2.add(link);
			}
			
			Object[] st1 = arrLink2.toArray();
			for (Object s : st1) {
				if (arrLink2.indexOf(s) != arrLink2.lastIndexOf(s)) {
					arrLink2.remove(arrLink2.lastIndexOf(s));
				}
			}
			
			String linkUpdate = "";
			for (int i = 0; i < arrLink2.size(); i++) {
				String link = arrLink2.get(i);
				
				if(link!=null&&!link.trim().equals(""))
				{
					if(linkUpdate.equals(""))
						linkUpdate = link;
					else
						linkUpdate += "|"+link;
				}

			}
			
			try {
				System.out.println(url);
				HTTPRequest request = new HTTPRequest(url, HTTPMethod.POST);
				
	            request.setHeader(new HTTPHeader("Content-Type", "application/x-www-form-urlencoded"));
	            String params = "link="+ Utils.EncodeString(listLink) +
	            				"&blog="+ Utils.EncodeString(blog)+
	            				"&sitemap="+ Utils.EncodeString(sitemap)+
	            				"&titleSelecter="+ Utils.EncodeString(titleSelecter)+
	            				"&contentSelecter="+ Utils.EncodeString(contentSelecter)+
	            				"&removeSelecter="+ Utils.EncodeString(removeSelecter)+
	            				"&customSelecter="+ Utils.EncodeString(customSelecter)+
	            				"&category="+ Utils.EncodeString(category)+
	            				"&typelink="+ Utils.EncodeString(typelink)+
	            				"&source="+ Utils.EncodeString(source)+
	            				"&sourcetype="+ Utils.EncodeString(sourcetype)+
	            				"&source_image="+ Utils.EncodeString(source_image)+
	            				"&remove_tag_a="+ Utils.EncodeString(remove_tag_a)+
	            				"&userBlog="+ Utils.EncodeString(userBlog)+
	            				"&reglink="+ Utils.EncodeString(reglink)+
	            				"&tag="+ Utils.EncodeString(tag)+
	            				"&linkSelecterJs="+ Utils.EncodeString(linkSelecterJs)+
	            				"&activeLinkJs="+ Utils.EncodeString(activeLinkJs)+
	            				"&linkSelecter="+ Utils.EncodeString(linkSelecter)+
	            				"&linkUpdate="+ Utils.EncodeString(linkUpdate)+
	            				"&activeContentJs="+ Utils.EncodeString(activeContentJs);
	            
	            request.setPayload(params.getBytes());
	            
	            fetcher.fetchAsync(request);
	            
			} catch (Exception e) {
				e.printStackTrace();
			}
		} else {
			URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
			
			URL url = new URL("http://hatforrent-home.appspot.com/change-time-craw");
			
			try {
				HTTPRequest request = new HTTPRequest(url, HTTPMethod.POST);
				
	            request.setHeader(new HTTPHeader("Content-Type", "application/x-www-form-urlencoded"));
	            String params = "userBlog="+ Utils.EncodeString(userBlog) +
	            				"&blog="+ Utils.EncodeString(blog)+
	            				"&sitemap="+ Utils.EncodeString(sitemap);
	            request.setPayload(params.getBytes());
	            
	            fetcher.fetchAsync(request);
	            
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
	}
}
